# -*- coding: utf-8 -*-
from jms.ods.tms.yl_tmsnew_branch_shipment import jms_ods__yl_tmsnew_branch_shipment
from jms.dm.dm_waybill_prescription_reach_details_platform_dt import jms_dm__dm_waybill_prescription_reach_details_platform_dt
from jms.dwd.dwd_terminal_sign_detail_dt import jms_dwd__dwd_terminal_sign_detail_dt
from jms.dwd.dwd_centersend_timerate_detail_dt import jms_dwd__dwd_centersend_timerate_detail_dt
from utils.operators.latest_only_bash_operator import LatestOnlyBashOperator
from utils.operators.latest_only_spark_submit_operator import LatestOnlySparkSubmitOperator


export__economic_circle_time_analyse_new = LatestOnlySparkSubmitOperator(
    task_id='export__economic_circle_time_analyse_new',
    pool_slots=5,
    # depends_on_past=True,  # 如果任务依赖于前一天的同名任务，则将 depends_on_past 设为 True
    task_concurrency=1,  # 如果任务不支持并发，则将 task_concurrency 设为 1
    name='export__economic_circle_time_analyse_new_{{ execution_date | cst_ds }}',  # yarn 任务名称
    driver_memory='4G',
    executor_memory='10G',
    executor_cores=6,
    num_executors=10,
    email=['zhangqinglin@jtexpress.com','yl_bigdata@yl-scm.com'],
    conf={'spark.executor.memoryOverhead'         : 2048,
          'spark.core.connection.ack.wait.timeout': 3000,
          'spark.default.parallelism'             : 600,
          'spark.locality.wait'                   : 60,
          },
    java_class='com.yunlu.bigdata.jobs.export.ExportEconomicCircleTimeAnalysenew2HDFS',  # spark 主类
    jars='hdfs:///scheduler/jms/spark/sjm/oss_flight_time_analyse/common-1.0-SNAPSHOT.jar',
    application='hdfs:///scheduler/jms/spark/sjm/oss_flight_time_analyse/original-jobs-1.0-SNAPSHOT.jar',  # spark jar 包
    application_args=['{{ execution_date  | date_add(-1) | cst_ds }}'],  # 参数dt 2020-10-26
)


upload_economic_circle_time_analyse_new = LatestOnlyBashOperator(
    task_id="upload_economic_circle_time_analyse_new",
    email=['zhangqinglin@jtexpress.com','yl_bigdata@yl-scm.com'],
    pool='unlimited_pool',
    bash_command='jms/oss/economic_circle_time_analyse_new/upload_economic_circle_time_analyse_new.sh',
)

# 设置依赖
upload_economic_circle_time_analyse_new << export__economic_circle_time_analyse_new << \
[jms_ods__yl_tmsnew_branch_shipment,
 jms_dm__dm_waybill_prescription_reach_details_platform_dt,
 jms_dwd__dwd_terminal_sign_detail_dt,
 jms_dwd__dwd_centersend_timerate_detail_dt
 ]
